from itertools import *

print "The main purpose of this program is to output the following: What percentage of queries contains the term pair"
print "begin..."

print "step1: load the query term first"
queryTermPairFreqDict = {}

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermPairNoDirectionProbabilityFor100KQueries_NEW_sorted_by_probability"
inputFileHandler = open(inputFileName,"r")

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTermIndexNum = lineElements[0]
    queryTermPair = lineElements[1]
    queryTermFreq = lineElements[2]
    queryTermProbability = lineElements[3]
    
    if queryTermPair not in queryTermPairFreqDict:
        queryTermPairFreqDict[queryTermPair] = 0

inputFileHandler.close()
print "len(queryTermPairFreqDict):",len(queryTermPairFreqDict)


print "step2: iterate all the 100K queries I have, for each of them, find their combinations. And add the corresponding key counter"
inputQueryFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueries"
inputQueryHandler = open(inputQueryFileName,"r")
for line in inputQueryHandler.readlines():
    elements = line.strip().split(":")
    queryID = int(elements[0])
    
    data = elements[1]
    data = data.lower()
    
    for i in range(0,len(data)):
        # print "data[i]:",ord(data[i])
        if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
            # Just replace them with a space.
            data = data[:i] + " " + data[i+1:]
    
    queryContent = data
    
    queryContentElements = queryContent.strip().split(" ")
    queryContentElementsWithoutEmptyStringAndWithoutDuplicateTerm1AndTerm2 = []
    
    for term in queryContentElements:
        if term.strip() != "" and term.strip() not in queryContentElementsWithoutEmptyStringAndWithoutDuplicateTerm1AndTerm2:
            queryContentElementsWithoutEmptyStringAndWithoutDuplicateTerm1AndTerm2.append(term)
    
    queryContentElementsWithoutEmptyStringAndWithoutDuplicateTerm1AndTerm2.sort(cmp=None, key=None, reverse=False)

    # for debug ONLY 
    # print queryContentElementsWithoutEmptyStringAndWithoutDuplicateTerm1AndTerm2
    # print len(queryContentElementsWithoutEmptyStringAndWithoutDuplicateTerm1AndTerm2)
    
    # print queryContentElementsWithoutEmptyStringAndWithoutDuplicateTerm1AndTerm2
    for currentTuple in combinations(queryContentElementsWithoutEmptyStringAndWithoutDuplicateTerm1AndTerm2,2):
        # print currentTuple
        (term1,term2) = currentTuple
        testedKey = term1 + "_" + term2
        if testedKey not in queryTermPairFreqDict:
            print "no such testedKey:",testedKey
            exit(1)
        else:
            # for debug and it is good.
            '''
            if testedKey == "19th_tb":
                print queryID
            '''
            queryTermPairFreqDict[testedKey] += 1

print "step3: output the file to the hard drive"

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermPairProbabilityOfTheQueryTrace_sorted_by_query_term_pair"
outputFileHandler = open(outputFileName,"w")

queryTermPairList = []
queryTermPairList = queryTermPairFreqDict.keys()
queryTermPairList.sort(cmp=None, key=None, reverse=False)
for indexNum,queryTermPair in enumerate( queryTermPairList ):
    # print indexNum,queryTermPair,queryTermPairFreqDict[queryTermPair]
    if queryTermPairFreqDict[queryTermPair] == 0:
        pass
    else:
        outputFileHandler.write( str(indexNum) + " " + queryTermPair + " " + str( queryTermPairFreqDict[queryTermPair] ) + "\n" )
outputFileHandler.close()
print "end."















